# -*- coding: UTF-8 -*-
import os
import requests
from tqdm import tqdm
proxies = {
  "http": "http://127.0.0.1:1087"
}

def imageDownload(url, folder):
    file_name = url.split("/")[-1]
    file_path = folder + "/target/" + file_name
    if os.path.exists(file_path):
        print(file_path, "is exists, skip")
        return

    print("Downloading %s from %s" % (file_name, url))
    try:
        download_file = requests.get(url,proxies=proxies,timeout=6)
        status = download_file.status_code
        if status == 200:
            with open(file_path, 'wb') as outfile:
                outfile.write(download_file.content)
                print("download success")
        else:
            print("download fail")
    except:
        print("download fail")

count = 0
rootdir = "raw_data/non-porn-nsfw"
for dirpath, dirnames, filenames in os.walk(rootdir):
    if len(filenames) > 0:
        for filename in filenames:
            suffix = filename.split(".")[-1]
            if suffix != "txt":
                continue
            path = os.path.join(dirpath, filename)
            with open(path, 'r') as f:
                lines = f.readlines()
                for url in tqdm(lines):
                    imageDownload(url.strip(), dirpath)







